#ifndef LEARNING_ENGINE_HIERARCHICAL_H
#define LEARNING_ENGINE_HIERARCHICAL_H

#include <random>
#include "learning_engine_base.h"
#include "feature_state.h"
#include "feature_knowledge.h"

enum HIERARCHICAl_LAYER {
    HIEREACHICAL_MANAGER = 0,
    HIERARCHICAL_WORKER,
    HIERARCHICAL_LAYER_MAX
};

enum ADDRESS_SECTION {
    ADDRESS_PAGE = 0,
    ADDRESS_OFFSET,
    ADDRESS_SECTION_MAX
};

enum ATROPOS_QTYPE {
    ATROPOS_PAGE_MANAGER = 0,
    ATROPOS_PAGE_WORKER,
    ATROPOS_OFFSET_MANAGER,
    ATROPOS_OFFSET_WORKER,
    ATROPOS_TYPE_MAX,
};

#define MANAGER_CNT 1
#define WORKER_CNT 8

typedef struct {
    uint32_t page_option;
	uint32_t page_action;
	uint32_t offset_option;
	uint32_t offset_action;
} Action;


/******************************************************************************/
// 等同于单learning_engine_featurewise
class FeatureGroup
{
private:
    FeatureKnowledge* m_feature_knowledges[NumFeatureTypes];

    /* tracing related knobs */
	uint32_t trace_interval;
	uint64_t trace_timestamp;
	FILE *trace;

private:
    float consultQ(State *state, uint32_t action);
    uint32_t getMaxAction(State *state, uint32_t option);

public:
    FeatureGroup(vector<int> &features, vector<int> &hash, float alpha, float gamma, uint32_t actions, bool zero_init);
    ~FeatureGroup();
    uint32_t chooseAction(State *state, uint32_t option);
    void learn(State *state1, uint32_t action1, int32_t reward, State *state2, uint32_t action2, RewardType reward_type);
};

/******************************************************************************/
// 等同于多个learning_engine_featurewise的分层组合
class LearningEngineHierarchical : public LearningEngineBase
{
private:
	FeatureGroup *m_PAGE_L1Q;
	FeatureGroup *m_PAGE_L2Q;
	FeatureGroup *m_OFFSET_L1Q;
	FeatureGroup *m_OFFSET_L2Q;

	/* tracing related knobs */
	uint32_t trace_interval;
	uint64_t trace_timestamp;
	FILE *trace;

private:
	void init_knobs();
	void init_stats();

    float consultQ(State *state, uint32_t action);
    uint32_t getMaxAction(State *state);

public:
	LearningEngineHierarchical(Prefetcher *p, float alpha, float gamma, float epsilon, uint32_t actions, uint64_t seed, std::string policy, std::string type, bool zero_init);
	~LearningEngineHierarchical();
    uint32_t choosePageOption(State *state, uint32_t option);
    uint32_t choosePageAction(State *state, uint32_t option);
    uint32_t chooseOffsetOption(State *state, uint32_t option);
    uint32_t chooseOffsetAction(State *state, uint32_t option);
    void learn(State *state1, Action &action1, int32_t reward, State *state2, Action &action2, RewardType reward_type);
    void dump_stats();
};

#endif /* LEARNING_ENGINE_HIERARCHICAL_H */
